/*
 *  proxy_hunter.cpp
 *  proxy_hunter
 *
 *  Created by Roy Zuo on May 13, 2010.
 *  Copyright 2010 O2 Micro, Inc. All rights reserved.
 *
 */


#include "proxy_hunter.hpp"

#include "unique_filename.hpp"

#include <sstream>
#include <iostream>

#include <boost/algorithm/string.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int.hpp>
#include <boost/random/variate_generator.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>

ProxyHunter::ProxyHunter ( const string& db_file, const string& protocol, const string& host, const vector< string >& paths, const HttpHeaderPairs& headers, const vector< string >& expressions )
: _protocol ( protocol ), _hostname ( host ), _headers ( headers ), _expressions ( expressions ), _paths ( paths ), _reader ( 0 )
{
	_reader = new ProxyReader ( db_file );
}

ProxyHunter::~ProxyHunter ()
{
	if ( _reader )
	{
		delete _reader;
		_reader = 0;
	}
}

void ProxyHunter::operator () ()
{	
	for ( vector< string >::const_iterator iter = _paths.begin(); iter != _paths.end(); iter ++ )
	{
		stringstream header_stream;
		stringstream content_stream;
		if ( read_webpage ( _protocol, _hostname, *iter, _headers, header_stream, content_stream ) )
		{
			parse_webpage ( header_stream, content_stream );
		}
	}
}

bool ProxyHunter::read_webpage( const string& protocol, const string& hostname, const string path, HttpHeaderPairs& headers, ostream& header_stream, ostream& content_stream )
{
	cout << "Reading < " << protocol << "://" << hostname << path << " > ...\n";

	boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();
	int retry = 3;
	while ( retry -- )
	{
		try
		{
			read_try ( retry, protocol, hostname, path, headers, header_stream, content_stream );
			retry = 0;
			boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
			boost::posix_time::time_duration td = end_time - start_time;
			cout << " OK! (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
			return true;
		}
		catch ( exception& ex )
		{
			if ( retry == 0 )
			{
				boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
				boost::posix_time::time_duration td = end_time - start_time;
				cout << " Failed ... (Use Time: " << boost::posix_time::to_simple_string ( td ) << ")\n";
				cerr << "\tException Occured: " << ex.what() << " >\n";
			}
		}
	}

	return false;
}

void ProxyHunter::read_try ( int count, const string& protocol, const string& hostname, const string path, HttpHeaderPairs& headers, ostream& header_stream, ostream& content_stream  )
{
	int times = 3 - count;
	if ( times == 1 )
	{
		cout << "  Trying ... the 1st time ...";
	}
	else if ( times == 2 )
	{
		cout << "  Trying ... the 2nd time ...";
	}
	else if ( times == 3 )
	{
		cout << "  Trying ... the 3rd time ...";
	}
	else
	{
		cout << "  Trying ... the " << times << "st time ...";
	}

	boost::asio::io_service io_service;
	WebpageHunter hunter ( io_service, &header_stream, &content_stream, hostname, protocol, hostname, path, headers );
	io_service.run();
}

void ProxyHunter::parse_webpage ( stringstream& header_stream, stringstream& content_stream )
{
	boost::posix_time::ptime start_time = boost::posix_time::second_clock::local_time();
	
	cout << "Parsing proxies from webpage ...\n";

	string page_header = header_stream.str ();
	string page_content = content_stream.str ();

	for ( vector< string >::const_iterator iter = _expressions.begin(); iter != _expressions.end(); iter ++ )
	{
		vector < string > proxies;
		search_proxies ( *iter, page_content, proxies );

		if ( proxies.size() > 0 )
		{
			_reader->Insert ( proxies, 0 );
		}
	}



	boost::posix_time::ptime end_time = boost::posix_time::second_clock::local_time();
	boost::posix_time::time_duration td = end_time - start_time;
	cout << "\tUse Time: " << boost::posix_time::to_simple_string ( td ) << "\n";
}

void ProxyHunter::search_proxies ( const string& expr, const string& page_content, vector < string >& proxies )
{
	string::const_iterator start, end;
	start = page_content.begin();
	end = page_content.end(); 

	boost::regex expression ( expr );
	boost::match_results<std::string::const_iterator> what;
	boost::match_flag_type flags = boost::match_default;
	while(boost::regex_search(start, end, what, expression, flags))   
	{
		string hostname = string ( what[1].first, what[1].second );
		string port = "80";
		if ( what.size() > 2 )
		{
			port = string ( what[2].first, what[2].second );
		}
		string proxy = "";
		_reader->MergeProxy ( hostname, port, proxy );

		cout << "  Found proxy: < " << proxy << " > ... \n";
		
		proxies.push_back ( proxy );

		// update search position:
		start = what[0].second;      
		// update flags:
		flags |= boost::match_prev_avail;
		flags |= boost::match_not_bob;
	}
}